#include "web/PageLibPreprocessor.hh"

#include <fstream>
#include <sstream>
#include <iostream>
#include <string>
#include <list>
#include <memory>


// 文档数量
#define N 3636

using std::cout;
using std::ifstream;
using std::ofstream;
using std::istringstream;
using std::string;
using std::list;
using std::unique_ptr;

using namespace tinyxml2;
using namespace simhash;

PageLibPreprocessor::PageLibPreprocessor(SplitTool* tool)
    : _tool(tool) {

}
void PageLibPreprocessor::doProcess(){
    readInfoFromFile();
}
void PageLibPreprocessor::readInfoFromFile(){
    // 偏移库
    readOffSet();
    cout << "偏移库读完,大小为: " << _offsetLib.size() << endl;
    // 网页库
    readRipePage();
    cout << "网页库读完,大小为: " << _pageLib.size() << endl;
}
void PageLibPreprocessor::cutRedundantPages(){
    // simhahs 计算指纹
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    Simhasher simhasher(conf_map["dict_path"][0], 
                        conf_map["model_path"][0], 
                        conf_map["idfPath"][0], 
                        conf_map["stopWordPath"][0]
                        );
    // 旧的指纹库 
    vector<pair<size_t, uint64_t>> fingerprint;
    createOldFingerprint(simhasher, fingerprint);

    // 指纹去重库， 去重后指纹没用了
    vector<pair<int, uint64_t>> new_fingerprint = createNewFingerprint(simhasher, fingerprint);

    // 构建去重索引库
    // NOTE: 需要修改 pos 和 offset 还没有修改
    map<int, pair<size_t, size_t>> new_index;
    creatNewIndex(new_fingerprint, new_index);
    cout << "new_index size: " << new_index.size() << endl;
    // 构建新的网页表
    createNewRipePage(new_index);
}
void PageLibPreprocessor::bulidInvertIndexTable(){
    
    cout << "bulidInvertIndexTable" << endl;
    // 得到 单词 和 文章 id 的映射表
    map<string, set<int>> ms; // 单词和文章 id 的映射
    getWordAndPageId(ms);
    cout << "bulidInvertIndexTable" << endl;
    // 给 invert index 表 单词值
    cout << "bulidInvertIndexTable" << endl;
    for(auto& elem : ms){
        _invertIndexTable[elem.first];
    }
    cout << "bulidInvertIndexTable" << endl;
    // 在遍历一遍 ms 的过程中，得到 invert index 表
    getTF(ms);
    cout << "bulidInvertIndexTable" << endl;
}
void PageLibPreprocessor::storeOnDisk(){
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    ofstream ofs(conf_map["invertIndex"][0]);
    for(auto& elem : _invertIndexTable){
        ofs << elem.first << " {";

        for(auto& elem2 : elem.second){
            ofs << elem2.first << ":" << elem2.second << ",";
        }
        ofs << "}\n";
    }

}


void PageLibPreprocessor::readOffSet(){
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    ifstream ifs;
    ifs.open(conf_map["offset"][0]);

    while(ifs){
        size_t docid;
        size_t pos;
        size_t offset;

        ifs >> docid;
        ifs >> pos;
        ifs >> offset;

        // cout << docid << " " << pos << " " << offset << "\n";
        _offsetLib.insert({docid, {pos, offset}});
    }
    ifs.close();



    // for(auto& elem : _offsetLib){
    //     cout << elem.first << " " << elem.second.first << " " << elem.second.second << "\n";
    // }
}

void PageLibPreprocessor::readRipePage(){
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    ifstream ifs(conf_map["ripepage"][0]);
    
    for(size_t i = 1; i <= _offsetLib.size(); ++i){
        auto page = _offsetLib[i];
        unique_ptr<char[]> buffer(new char[page.second + 1]);

        ifs.seekg(page.first, ios::beg);
        ifs.read(buffer.get(), page.second);
    
        XMLDocument doc;
        doc.Parse(buffer.get(), page.second);
        XMLElement* root = doc.RootElement();
        if(root == nullptr){
            cout << "解析失败\n";
            continue;
        }

        XMLElement* Xmldocid = root->FirstChildElement("docid");
        XMLElement* Xmltitle = root->FirstChildElement("title");
        XMLElement* Xmlurl =  root->FirstChildElement("url");
        if(Xmldocid == nullptr || Xmltitle == nullptr || Xmlurl == nullptr){
            cout << "解析失败\n";
            continue;
        }
        string docid = Xmldocid->GetText();
        string title = Xmltitle->GetText();
        string url = Xmlurl->GetText();
        // cout << docid << " " << title << " " << url << endl;

        _pageLib.push_back({title, url});

    } 
}



void PageLibPreprocessor::createOldFingerprint(Simhasher& simhasher, vector<pair<size_t, uint64_t>>& fingerprint){
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    XMLDocument doc;
    doc.LoadFile(conf_map["ripepage"][0].c_str());

    XMLElement* root = doc.RootElement();
    for(XMLElement* p = root; p != nullptr; p = p->NextSiblingElement()){
        string docid = p->FirstChildElement("docid")->GetText();
        string content = p->FirstChildElement("content")->GetText();

        size_t topN = 5;
        uint64_t u64 = 0;
        simhasher.make(content, topN, u64);
        fingerprint.push_back({atoi(docid.c_str()), u64});
    }
}

vector<pair<int, uint64_t>> PageLibPreprocessor::createNewFingerprint(Simhasher& simhasher, vector<pair<size_t, uint64_t>>& fingerprint){
    vector<pair<int, uint64_t>> new_fingerprint;

    for(size_t i = 0; i < fingerprint.size(); i++){
        size_t j = i + 1;
        for(; j < fingerprint.size(); j++){

            if(Simhasher::isEqual(fingerprint[i].second, fingerprint[j].second)){
                break;
            }
        }
        if(j == fingerprint.size()){
            new_fingerprint.push_back(fingerprint[i]);
        }
    }
    return new_fingerprint;
}

void PageLibPreprocessor::creatNewIndex(vector<pair<int, uint64_t>>& new_fingerprint, map<int, pair<size_t, size_t>>& new_index){
    for(auto& elem : new_fingerprint){
        new_index.insert({elem.first, {_offsetLib[elem.first].first, _offsetLib[elem.first].second}});
    }
}

void PageLibPreprocessor::createNewRipePage(map<int, pair<size_t, size_t>>& new_index){
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    ifstream ifs(conf_map["ripepage"][0]);
    ofstream ofs(conf_map["newripepage"][0]);
    
    ofstream ofst;
    ofst.open(conf_map["newoffset"][0], std::ios::ate);

    for(auto& elem : new_index){
        size_t pos = elem.second.first;
        ifs.seekg(pos, std::ios::beg);
        size_t len = elem.second.second;
        unique_ptr<char[]> buffer(new char[len + 1] ());
        ifs.read(buffer.get(), len);
        
        size_t old_size = ofs.tellp();

        ofs.write(buffer.get(), len);
        size_t new_size = ofs.tellp();

        ofst << elem.first << " " << old_size << " " << new_size - old_size << endl;
    }
}


void PageLibPreprocessor::getWordAndPageId(map<string, set<int>>& ms){
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    XMLDocument doc;
    doc.LoadFile(conf_map["newripepage"][0].c_str());

    XMLElement* root = doc.RootElement();
    for(XMLElement* p = root; p != nullptr; p = p->NextSiblingElement()){
        string docid = p->FirstChildElement("docid")->GetText();
        string content = p->FirstChildElement("content")->GetText();
        cout << "id = " << docid << "\n";

        // 分词
        vector<string> v = _tool->cut(content);

        set<string> s;
        for(auto& elem : v){
            s.insert(elem);
        }

        for(auto& elem : s){
            ms[elem].insert(atoi(docid.c_str()));
        }
    }
}


void PageLibPreprocessor::getTF(map<string, set<int>>& ms){
    auto& conf_map = Configuration::createConfig()->getConfigMap();
    XMLDocument doc;
    doc.LoadFile(conf_map["newripepage"][0].c_str());

    XMLElement* root = doc.RootElement();
    for(XMLElement* p = root; p != nullptr; p = p->NextSiblingElement()){
        string docid = p->FirstChildElement("docid")->GetText();
        string content = p->FirstChildElement("content")->GetText();
        cout << "id = " << docid << "\n";
        // 分词
        vector<string> v = _tool->cut(content);

        map<string, int> m; // 单篇文章的词频
        for(auto& elem : v){
            if(m.find(elem) == m.end()){
                // 新词
                m[elem] = 1;
            }else{
                m[elem]++;
            }
        }

        double sum = 0;
        for(auto& elem : m){
            double TF = elem.second;
            double DF = ms[elem.first].size();
            double IDF = log2(N / (DF + 1));

            sum += pow(TF * IDF, 2);
        }

        // 第二次遍历要做归一化处理
        double norm = sqrt(sum);

        for(auto& elem : m){
            double TF = elem.second;
            double DF = ms[elem.first].size();
            double IDF = log2(N / (DF + 1));
            
            double w = (TF * IDF) / norm;
            
            _invertIndexTable[elem.first].push_back({atoi(docid.c_str()), w});
    
        }

    } 
}